"""hw0.py  ejf July 8,2013
 Read digitization mwe.txt.
 Output all major headwords, along with the page on which the headword appear.
 Also, output the line numbers in inm.txt that pertain to the headword.
 - Page numbers are 
[Page-ppp-c+ n]
where
  ppp is 3-digit page number (pagination starts anew for each volume)
  c = a or b

 - Headwords are as in headword.py
   X is coded in HK encoding 
   It is checked that 'X' contains no colon character.
 - line numbers l1,l2 are the first line number and last line number of 
   ap.txt that pertain to the headword.  
   The 'end-of-volume' information is skipped (although present in the
   digitization).
 The output is written as  (note: 'page' is v-ppp,c)
   page:headword:line1,line2
 Change \x8C (LATIN CAPITAL LIGATURE OE) to oe. (23 headwords)
"""
import re
import sys
filename=sys.argv[1] # 
fileout =sys.argv[2] #  
f = open(filename,'r')
fout = open(fileout,'w')
n = 0
page='001-a'

nb = 0 # number of left brackets
nout = 0 # number of headword lines written to output
rePage = re.compile(r"\[Page(.*?)]")
import headword
reHeadword0 = headword.reHeadword
reHeadword = re.compile(reHeadword0)
l0=0 # first line number for a headword
nhw=0 # same as n, but stops when the 'end' string is found
#first line of file, not processed 
firststring=r'[Page001-a+ 46]'
firstfound=False
endstring=r'XXXXXXXXXX'  #not really needed. file is read to end
endfound=False
isFirst = False
outlines = []
for line in f:
 n = n+1
 if (line.find(endstring) >= 0): 
  endfound=True
 nhw = nhw + 1
 if (line.find(firststring) >= 0): 
  print "found firststring: %s" % firststring
  firstfound=True
  # DO process this line (to get line number)
  isFirst = False 
  print "found firststring"
 # the placement of m=.. before if(not firstfound) is important detail
 # Change 
 line1 = re.sub(r'\x8C','OE',line)
 if (line1 != line):
  #print "changed line:\nOLD=%s\nNEW=%s\n" %(line,line1)
  line= line1
 m = reHeadword.search(line)
 if (not firstfound):
  continue
 if m and (not isFirst):
  # found next headword
  if (l0 != 0):
   # output the prior word
   l1 = l0
   l2 = nhw - 1
   out = "%s:%s:%s,%s" %(page0,hw0,l1,l2)
   outlines.append(out)
   #fout.write("%s\n" %(out,))
   nout = nout + 1
  # the base headword. This program outputs this
  hw00 = m.group(1) 
  # get hw0 from hw00.
  # remove leading {%To%}. Convert to lower case
  hw0 = re.sub(r'\{%To%\} *','',hw00)
  hw0 = hw0.lower()
  page0 = page
  l0=nhw
  if re.search(":",hw0):
   print "Removing from ':' to end at line %s: %s" % (n,hw0)
   nb = nb + 1
   hw0 = re.sub(':.*$','',hw0)
 # step 3, search for page
 isFirst = False # required for first word handling
 pages = rePage.findall(line)
 if len(pages) > 0:
  pagelast = pages[-1]
  m = re.match(r'^([0-9]+)-([ab])[+]',pagelast)
  if m:
   page = "%s-%s" % (m.group(1),m.group(2))
 if endfound:
  break
# we must now prepare the last headword
l1 = l0
l2 = nhw
out = "%s:%s:%s,%s" %(page0,hw0,l1,l2)
outlines.append(out)
nout = nout + 1
#---------Adjust outlines.
# Specifically, adjust the last line in each volume, as needed
# This is done 'manually'
# nlasts = array, last line in outlines for given volume;
#  the numbering starts at 1.
#  There are actually 48236 lines in outlines. But, the
#  dictionary ends at # 48231. So, 48232 - 48236 are deleted.
# lastlines = array, last line in ap.txt for the corresponding outlines 
#  entry.
if False:  # skip this logic
 nlasts = [1874,3843] 
 lastlines = [27049,52134]
 for i in range(0,2):
  nlast = nlasts[i]
  lastline = lastlines[i]
  out = outlines[nlast - 1]
  (page,word,lines) = re.split(':',out)
  (line1,line2) = re.split(',',lines)
  line2 = lastline
  lines = "%s,%s" %(line1,line2)
  out = "%s:%s:%s" %(page,word,lines)
  outlines[nlast - 1] = out
 
# remove non-dictionary end-of-volume lines
outlines1 = []
for i in xrange(0,nout):
 j = i + 1
 if False: # skip this adjustment logic 
  if (3844 <= j) and (j <= 3906):
   continue
  if (1875 <= j) and (j <= 1884):
   continue
 outlines1.append(outlines[i])

#---------Output adjusted lines
nout = 0
for out in outlines1:
 fout.write("%s\n" %(out,))
 nout = nout + 1
f.close();
fout.close();
print "file %s has %s lines" % (filename,n)
print "%s headwords written to file %s" % (nout,fileout)
print "%s headwords contained a colon" % (nb,)
